import pandas as pd
import numpy as np
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
import plotly.express as px
import plotly.graph_objs as go
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error
from sklearn.model_selection import train_test_split

from math import sqrt

def smape(A, F):
    return 100/len(A) * np.sum(2 * np.abs(F - A) / (np.abs(A) + np.abs(F)))


def vape(y_true, y_pred):
    return np.mean(np.abs(y_true - y_pred))/np.mean(np.abs(y_true))

clicks_df = pd.read_csv("click_for_day.csv").drop(["Unnamed: 0","pred","date_obj"], axis=1)

clicks_df['date'] = pd.to_datetime(clicks_df['date']).dt.date

clicks_df.head()

fig = px.line(clicks_df, x="date", y="clicks", title='Count of clicks for a day')
fig.show()

weather_df = pd.read_csv("weather_history.csv")

weather_df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 43056 entries, 0 to 43055
Data columns (total 27 columns):
 #   Column          Non-Null Count  Dtype  
---  ------          --------------  -----  
 0   city            43056 non-null  object 
 1   region          43056 non-null  object 
 2   country         43056 non-null  object 
 3   latitude        0 non-null      float64
 4   longitude       0 non-null      float64
 5   datetime        43056 non-null  object 
 6   maxtempC        43056 non-null  int64  
 7   mintempC        43056 non-null  int64  
 8   avgtempC        43056 non-null  int64  
 9   totalSnow_cm    43056 non-null  float64
 10  sunHour         43056 non-null  float64
 11  tempC           43056 non-null  int64  
 12  windspeedKmph   43056 non-null  int64  
 13  winddirDegree   43056 non-null  int64  
 14  winddir16Point  43056 non-null  object 
 15  weatherCode     43056 non-null  int64  
 16  weatherDesc     0 non-null      float64
 17  precipMM        43056 non-null  float64
 18  humidity        43056 non-null  int64  
 19  visibility      43056 non-null  int64  
 20  pressure        43056 non-null  int64  
 21  cloudcover      43056 non-null  int64  
 22  HeatIndexC      43056 non-null  int64  
 23  WindChillC      43056 non-null  int64  
 24  WindGustKmph    43056 non-null  int64  
 25  FeelsLikeC      43056 non-null  int64  
 26  uvIndex         43056 non-null  int64  
dtypes: float64(6), int64(16), object(5)
memory usage: 8.9+ MB

weather_df.describe()

weather_df['datetime'] = pd.to_datetime(weather_df['datetime'])

weather_df['date'] = weather_df['datetime'].dt.date

weather_df.head()

weather_df['city'].value_counts()

city
Calgary    43056
Name: count, dtype: int64

weather_df.columns

Index(['city', 'region', 'country', 'latitude', 'longitude', 'datetime',
       'maxtempC', 'mintempC', 'avgtempC', 'totalSnow_cm', 'sunHour', 'tempC',
       'windspeedKmph', 'winddirDegree', 'winddir16Point', 'weatherCode',
       'weatherDesc', 'precipMM', 'humidity', 'visibility', 'pressure',
       'cloudcover', 'HeatIndexC', 'WindChillC', 'WindGustKmph', 'FeelsLikeC',
       'uvIndex', 'date'],
      dtype='object')

weather_df.drop(['city', 'region', 'country', 'latitude', 'longitude', 'datetime',"winddir16Point","weatherDesc"], axis=1, inplace=True)

weather_df_by_date = weather_df.groupby(['date'], as_index=False).mean()

weather_df

df = pd.merge(clicks_df, weather_df_by_date, on='date', how='left', indicator=True)

df['_merge'].value_counts()

_merge
both          1758
left_only       89
right_only       0
Name: count, dtype: int64

df.dropna(inplace=True)

df['_merge'].value_counts()

_merge
both          1758
left_only        0
right_only       0
Name: count, dtype: int64

from ydata_profiling import ProfileReport

df.drop(['date',"_merge"], axis=1).corr()

profile = ProfileReport(df.drop(['date',"_merge"], axis=1),
                        title='Pandas Profiling Report')

profile

Summarize dataset:   0%|          | 0/5 [00:00<?, ?it/s]

Generate report structure:   0%|          | 0/1 [00:00<?, ?it/s]

Render HTML:   0%|          | 0/1 [00:00<?, ?it/s]

import lightgbm as lgb
from lightgbm import early_stopping
from lightgbm import log_evaluation

df['date'].max()

datetime.date(2023, 2, 3)

df['date'] = pd.to_datetime(df['date'])

train_df = df[df['date'] <= pd.to_datetime("2023-01-01")].copy()
test_df = df[df['date'] > pd.to_datetime("2023-01-01")].copy()

y_train = train_df['clicks']
X_train= train_df.drop(['clicks',"date","_merge"], axis=1)
X_test =test_df.drop(['clicks',"date","_merge"], axis=1)
y_test = test_df['clicks']

train_parameters = {
  "boosting_type": "gbdt",
  "num_leaves": 5,
  "max_depth": 100,
  "learning_rate": 0.001,
  "n_estimators": 10000,
  "subsample_for_bin": 40000,
  "objective": "regression",
  "min_split_gain": 0.36004551515184435,
  "min_child_weight": 6.680517668416932,
  "min_child_samples": 12,
  "subsample": 0.8113890445491936,
  "subsample_freq": 99,
  "colsample_bytree": 0.5100181540136696,
  "reg_alpha": 27.569704821799697,
  "reg_lambda": 95.00039766093927,
  "n_jobs": -1,

  "importance_type": "split",
    #Shows debug and warning level information (silent - is depricated) , verbose as parameter
      #"silent": True,
    # -1 value means - off some output
    "verbose":0,
    "force_col_wise" : True
}
fit_params={
            "eval_metric" : ['l1','l2'],
            "eval_set" : [(X_test,y_test)],
            'eval_names': ['error'],
            'feature_name': 'auto', # that's actually the default
            'categorical_feature': 'auto' # that's actually the default
           }
num_round =50

lgg = lgb.LGBMRegressor(**train_parameters)

model = lgg.fit(X_train, 
                y_train, 
                callbacks=[early_stopping(100),log_evaluation(10)],
                **fit_params)

Training until validation scores don't improve for 100 rounds
[10]	error's l1: 142.391	error's l2: 21603.6
[20]	error's l1: 142.185	error's l2: 21543.5
[30]	error's l1: 141.984	error's l2: 21485.4
[40]	error's l1: 141.625	error's l2: 21382.6
[50]	error's l1: 141.24	error's l2: 21273
[60]	error's l1: 141.13	error's l2: 21240.4
[70]	error's l1: 140.874	error's l2: 21167.7
[80]	error's l1: 140.547	error's l2: 21070.1
[90]	error's l1: 140.184	error's l2: 20961.4
[100]	error's l1: 140.023	error's l2: 20912.9
[110]	error's l1: 139.825	error's l2: 20853.6
[120]	error's l1: 139.612	error's l2: 20789.6
[130]	error's l1: 139.465	error's l2: 20745.4
[140]	error's l1: 139.062	error's l2: 20625.6
[150]	error's l1: 138.851	error's l2: 20563
[160]	error's l1: 138.453	error's l2: 20445
[170]	error's l1: 138.139	error's l2: 20352.2
[180]	error's l1: 137.682	error's l2: 20217.6
[190]	error's l1: 137.438	error's l2: 20145.9
[200]	error's l1: 137.217	error's l2: 20080.7
[210]	error's l1: 137.073	error's l2: 20037.9
[220]	error's l1: 136.878	error's l2: 19980.6
[230]	error's l1: 136.804	error's l2: 19957.8
[240]	error's l1: 136.395	error's l2: 19838.6
[250]	error's l1: 136.083	error's l2: 19747.6
[260]	error's l1: 135.969	error's l2: 19714
[270]	error's l1: 135.623	error's l2: 19613.7
[280]	error's l1: 135.47	error's l2: 19568.8
[290]	error's l1: 135.252	error's l2: 19505.3
[300]	error's l1: 134.969	error's l2: 19423.6
[310]	error's l1: 134.469	error's l2: 19280.2
[320]	error's l1: 133.978	error's l2: 19140.2
[330]	error's l1: 133.772	error's l2: 19081.5
[340]	error's l1: 133.533	error's l2: 19012.9
[350]	error's l1: 133.337	error's l2: 18956.8
[360]	error's l1: 133.137	error's l2: 18899.6
[370]	error's l1: 132.945	error's l2: 18844.7
[380]	error's l1: 132.751	error's l2: 18789.2
[390]	error's l1: 132.551	error's l2: 18732.2
[400]	error's l1: 132.348	error's l2: 18674.9
[410]	error's l1: 132.102	error's l2: 18606.1
[420]	error's l1: 131.951	error's l2: 18562.9
[430]	error's l1: 131.672	error's l2: 18484.3
[440]	error's l1: 131.491	error's l2: 18433.4
[450]	error's l1: 131.238	error's l2: 18362.3
[460]	error's l1: 130.976	error's l2: 18288.9
[470]	error's l1: 130.897	error's l2: 18266.5
[480]	error's l1: 130.756	error's l2: 18227
[490]	error's l1: 130.499	error's l2: 18155.3
[500]	error's l1: 130.237	error's l2: 18082.5
[510]	error's l1: 129.863	error's l2: 17979.2
[520]	error's l1: 129.754	error's l2: 17948.6
[530]	error's l1: 129.45	error's l2: 17865.1
[540]	error's l1: 129.195	error's l2: 17794.9
[550]	error's l1: 129.024	error's l2: 17747.7
[560]	error's l1: 128.658	error's l2: 17647.7
[570]	error's l1: 128.697	error's l2: 17657.7
[580]	error's l1: 128.524	error's l2: 17609.9
[590]	error's l1: 128.396	error's l2: 17574.4
[600]	error's l1: 128.25	error's l2: 17534.4
[610]	error's l1: 127.858	error's l2: 17427.9
[620]	error's l1: 127.556	error's l2: 17346.1
[630]	error's l1: 127.449	error's l2: 17316.8
[640]	error's l1: 127.348	error's l2: 17289.2
[650]	error's l1: 127.353	error's l2: 17289.9
[660]	error's l1: 127.091	error's l2: 17219.1
[670]	error's l1: 126.98	error's l2: 17188.6
[680]	error's l1: 126.745	error's l2: 17125.1
[690]	error's l1: 126.612	error's l2: 17088.7
[700]	error's l1: 126.453	error's l2: 17045.8
[710]	error's l1: 126.322	error's l2: 17010.6
[720]	error's l1: 126.187	error's l2: 16974.1
[730]	error's l1: 126.07	error's l2: 16942.4
[740]	error's l1: 125.766	error's l2: 16861.1
[750]	error's l1: 125.488	error's l2: 16787.3
[760]	error's l1: 125.357	error's l2: 16752.2
[770]	error's l1: 125.129	error's l2: 16691.7
[780]	error's l1: 124.96	error's l2: 16646.7
[790]	error's l1: 124.73	error's l2: 16585.9
[800]	error's l1: 124.537	error's l2: 16534.7
[810]	error's l1: 124.14	error's l2: 16430.4
[820]	error's l1: 123.904	error's l2: 16368.5
[830]	error's l1: 123.837	error's l2: 16350.3
[840]	error's l1: 123.563	error's l2: 16278.9
[850]	error's l1: 123.326	error's l2: 16216.8
[860]	error's l1: 123.203	error's l2: 16184.3
[870]	error's l1: 122.875	error's l2: 16099.1
[880]	error's l1: 122.567	error's l2: 16019.1
[890]	error's l1: 122.434	error's l2: 15984.2
[900]	error's l1: 122.181	error's l2: 15918.6
[910]	error's l1: 121.957	error's l2: 15860.6
[920]	error's l1: 121.831	error's l2: 15828
[930]	error's l1: 121.666	error's l2: 15785.4
[940]	error's l1: 121.512	error's l2: 15745.6
[950]	error's l1: 121.402	error's l2: 15717.1
[960]	error's l1: 121.197	error's l2: 15664.4
[970]	error's l1: 120.996	error's l2: 15612.9
[980]	error's l1: 120.902	error's l2: 15588.5
[990]	error's l1: 120.719	error's l2: 15541.4
[1000]	error's l1: 120.62	error's l2: 15515.8
[1010]	error's l1: 120.386	error's l2: 15456.3
[1020]	error's l1: 120.236	error's l2: 15417.8
[1030]	error's l1: 120.008	error's l2: 15359.8
[1040]	error's l1: 119.769	error's l2: 15298.7
[1050]	error's l1: 119.631	error's l2: 15263.9
[1060]	error's l1: 119.531	error's l2: 15238.4
[1070]	error's l1: 119.377	error's l2: 15199.4
[1080]	error's l1: 119.177	error's l2: 15148.8
[1090]	error's l1: 119.16	error's l2: 15144
[1100]	error's l1: 119.049	error's l2: 15116
[1110]	error's l1: 118.824	error's l2: 15059.4
[1120]	error's l1: 118.615	error's l2: 15006.6
[1130]	error's l1: 118.371	error's l2: 14944.9
[1140]	error's l1: 118.099	error's l2: 14877
[1150]	error's l1: 117.859	error's l2: 14816.9
[1160]	error's l1: 117.555	error's l2: 14741.2
[1170]	error's l1: 117.624	error's l2: 14757.7
[1180]	error's l1: 117.472	error's l2: 14719.7
[1190]	error's l1: 117.263	error's l2: 14667.8
[1200]	error's l1: 117.147	error's l2: 14639.1
[1210]	error's l1: 116.891	error's l2: 14576
[1220]	error's l1: 116.846	error's l2: 14564.7
[1230]	error's l1: 116.697	error's l2: 14528
[1240]	error's l1: 116.579	error's l2: 14498
[1250]	error's l1: 116.505	error's l2: 14479.3
[1260]	error's l1: 116.205	error's l2: 14405.3
[1270]	error's l1: 115.988	error's l2: 14352.3
[1280]	error's l1: 115.926	error's l2: 14337.1
[1290]	error's l1: 115.819	error's l2: 14310.2
[1300]	error's l1: 115.552	error's l2: 14245
[1310]	error's l1: 115.379	error's l2: 14202.9
[1320]	error's l1: 115.13	error's l2: 14142.4
[1330]	error's l1: 114.871	error's l2: 14079.2
[1340]	error's l1: 114.697	error's l2: 14037
[1350]	error's l1: 114.511	error's l2: 13991.6
[1360]	error's l1: 114.309	error's l2: 13942.2
[1370]	error's l1: 114.091	error's l2: 13889.7
[1380]	error's l1: 113.95	error's l2: 13855.3
[1390]	error's l1: 113.784	error's l2: 13815.3
[1400]	error's l1: 113.58	error's l2: 13766.6
[1410]	error's l1: 113.452	error's l2: 13736.1
[1420]	error's l1: 113.337	error's l2: 13708.5
[1430]	error's l1: 113.211	error's l2: 13678.5
[1440]	error's l1: 113.101	error's l2: 13652.5
[1450]	error's l1: 112.856	error's l2: 13594.2
[1460]	error's l1: 112.681	error's l2: 13552.8
[1470]	error's l1: 112.485	error's l2: 13506.4
[1480]	error's l1: 112.273	error's l2: 13456.7
[1490]	error's l1: 112.176	error's l2: 13433.4
[1500]	error's l1: 112.142	error's l2: 13425
[1510]	error's l1: 112.097	error's l2: 13413.4
[1520]	error's l1: 111.997	error's l2: 13389.4
[1530]	error's l1: 111.844	error's l2: 13353.2
[1540]	error's l1: 111.751	error's l2: 13331.2
[1550]	error's l1: 111.729	error's l2: 13325.9
[1560]	error's l1: 111.647	error's l2: 13306.5
[1570]	error's l1: 111.456	error's l2: 13261.8
[1580]	error's l1: 111.328	error's l2: 13232
[1590]	error's l1: 111.282	error's l2: 13220.2
[1600]	error's l1: 111.039	error's l2: 13163.5
[1610]	error's l1: 110.879	error's l2: 13125.6
[1620]	error's l1: 110.787	error's l2: 13104.1
[1630]	error's l1: 110.584	error's l2: 13056.7
[1640]	error's l1: 110.382	error's l2: 13009.7
[1650]	error's l1: 110.152	error's l2: 12955.7
[1660]	error's l1: 109.964	error's l2: 12911.8
[1670]	error's l1: 109.831	error's l2: 12880.4
[1680]	error's l1: 109.581	error's l2: 12822.8
[1690]	error's l1: 109.448	error's l2: 12792.3
[1700]	error's l1: 109.288	error's l2: 12754.4
[1710]	error's l1: 109.11	error's l2: 12713.9
[1720]	error's l1: 108.964	error's l2: 12680.2
[1730]	error's l1: 108.882	error's l2: 12661.1
[1740]	error's l1: 108.718	error's l2: 12623.7
[1750]	error's l1: 108.583	error's l2: 12592.8
[1760]	error's l1: 108.357	error's l2: 12541.1
[1770]	error's l1: 108.202	error's l2: 12505.4
[1780]	error's l1: 108.027	error's l2: 12465.6
[1790]	error's l1: 107.921	error's l2: 12441.1
[1800]	error's l1: 107.728	error's l2: 12397.2
[1810]	error's l1: 107.595	error's l2: 12366.8
[1820]	error's l1: 107.368	error's l2: 12315.6
[1830]	error's l1: 107.351	error's l2: 12311.2
[1840]	error's l1: 107.215	error's l2: 12280.3
[1850]	error's l1: 107.118	error's l2: 12258
[1860]	error's l1: 106.983	error's l2: 12226.8
[1870]	error's l1: 106.818	error's l2: 12189.4
[1880]	error's l1: 106.757	error's l2: 12174.5
[1890]	error's l1: 106.63	error's l2: 12145.4
[1900]	error's l1: 106.432	error's l2: 12100.2
[1910]	error's l1: 106.357	error's l2: 12082.4
[1920]	error's l1: 106.205	error's l2: 12048.2
[1930]	error's l1: 106.054	error's l2: 12013.9
[1940]	error's l1: 105.831	error's l2: 11963.6
[1950]	error's l1: 105.719	error's l2: 11937.9
[1960]	error's l1: 105.586	error's l2: 11908.3
[1970]	error's l1: 105.494	error's l2: 11888
[1980]	error's l1: 105.301	error's l2: 11845.2
[1990]	error's l1: 105.206	error's l2: 11823.5
[2000]	error's l1: 105.095	error's l2: 11798.2
[2010]	error's l1: 105.034	error's l2: 11784.5
[2020]	error's l1: 104.999	error's l2: 11775.9
[2030]	error's l1: 104.863	error's l2: 11745
[2040]	error's l1: 104.806	error's l2: 11731.8
[2050]	error's l1: 104.792	error's l2: 11727.4
[2060]	error's l1: 104.683	error's l2: 11702.9
[2070]	error's l1: 104.565	error's l2: 11676.5
[2080]	error's l1: 104.478	error's l2: 11656.7
[2090]	error's l1: 104.371	error's l2: 11633.3
[2100]	error's l1: 104.284	error's l2: 11614.5
[2110]	error's l1: 104.22	error's l2: 11600.3
[2120]	error's l1: 104.144	error's l2: 11583.7
[2130]	error's l1: 104.013	error's l2: 11555.4
[2140]	error's l1: 103.884	error's l2: 11527.3
[2150]	error's l1: 103.715	error's l2: 11490.2
[2160]	error's l1: 103.622	error's l2: 11469.9
[2170]	error's l1: 103.494	error's l2: 11442.5
[2180]	error's l1: 103.353	error's l2: 11411.9
[2190]	error's l1: 103.228	error's l2: 11385.1
[2200]	error's l1: 103.06	error's l2: 11348.8
[2210]	error's l1: 102.927	error's l2: 11320.1
[2220]	error's l1: 102.743	error's l2: 11280.6
[2230]	error's l1: 102.604	error's l2: 11250.5
[2240]	error's l1: 102.47	error's l2: 11221.7
[2250]	error's l1: 102.346	error's l2: 11195.4
[2260]	error's l1: 102.269	error's l2: 11178.8
[2270]	error's l1: 102.16	error's l2: 11155.4
[2280]	error's l1: 102.039	error's l2: 11129.3
[2290]	error's l1: 101.929	error's l2: 11105.4
[2300]	error's l1: 101.842	error's l2: 11086.1
[2310]	error's l1: 101.708	error's l2: 11057.5
[2320]	error's l1: 101.563	error's l2: 11026.4
[2330]	error's l1: 101.451	error's l2: 11001.9
[2340]	error's l1: 101.286	error's l2: 10966.8
[2350]	error's l1: 101.122	error's l2: 10932
[2360]	error's l1: 100.991	error's l2: 10904.1
[2370]	error's l1: 100.907	error's l2: 10885.8
[2380]	error's l1: 100.739	error's l2: 10850.1
[2390]	error's l1: 100.638	error's l2: 10828.2
[2400]	error's l1: 100.467	error's l2: 10791.8
[2410]	error's l1: 100.287	error's l2: 10753.6
[2420]	error's l1: 100.08	error's l2: 10710.5
[2430]	error's l1: 99.9479	error's l2: 10682.3
[2440]	error's l1: 99.7654	error's l2: 10644
[2450]	error's l1: 99.5683	error's l2: 10602.8
[2460]	error's l1: 99.4499	error's l2: 10577.3
[2470]	error's l1: 99.3197	error's l2: 10549.7
[2480]	error's l1: 99.2221	error's l2: 10528.9
[2490]	error's l1: 99.0405	error's l2: 10490.9
[2500]	error's l1: 98.9118	error's l2: 10463.7
[2510]	error's l1: 98.8116	error's l2: 10442.6
[2520]	error's l1: 98.6472	error's l2: 10408.6
[2530]	error's l1: 98.4833	error's l2: 10374.8
[2540]	error's l1: 98.3175	error's l2: 10340.4
[2550]	error's l1: 98.1841	error's l2: 10312.6
[2560]	error's l1: 98.029	error's l2: 10280.2
[2570]	error's l1: 97.947	error's l2: 10262.2
[2580]	error's l1: 97.8604	error's l2: 10243.8
[2590]	error's l1: 97.7625	error's l2: 10222.3
[2600]	error's l1: 97.6321	error's l2: 10195.3
[2610]	error's l1: 97.4983	error's l2: 10166.7
[2620]	error's l1: 97.3472	error's l2: 10135.5
[2630]	error's l1: 97.1874	error's l2: 10102.4
[2640]	error's l1: 97.0529	error's l2: 10074.5
[2650]	error's l1: 96.976	error's l2: 10058.4
[2660]	error's l1: 96.8424	error's l2: 10030.8
[2670]	error's l1: 96.7749	error's l2: 10016.4
[2680]	error's l1: 96.7555	error's l2: 10012.2
[2690]	error's l1: 96.643	error's l2: 9989.63
[2700]	error's l1: 96.5657	error's l2: 9973.97
[2710]	error's l1: 96.5089	error's l2: 9962.54
[2720]	error's l1: 96.389	error's l2: 9937.8
[2730]	error's l1: 96.2738	error's l2: 9914.39
[2740]	error's l1: 96.2037	error's l2: 9899.59
[2750]	error's l1: 96.0944	error's l2: 9877.15
[2760]	error's l1: 96.0262	error's l2: 9863.02
[2770]	error's l1: 95.9872	error's l2: 9854.95
[2780]	error's l1: 95.8981	error's l2: 9836.6
[2790]	error's l1: 95.7723	error's l2: 9810.73
[2800]	error's l1: 95.6469	error's l2: 9785.28
[2810]	error's l1: 95.5367	error's l2: 9763.28
[2820]	error's l1: 95.3876	error's l2: 9733.75
[2830]	error's l1: 95.2561	error's l2: 9707.26
[2840]	error's l1: 95.1524	error's l2: 9686.28
[2850]	error's l1: 95.0926	error's l2: 9673.57
[2860]	error's l1: 94.9533	error's l2: 9645.71
[2870]	error's l1: 94.8242	error's l2: 9619.82
[2880]	error's l1: 94.7375	error's l2: 9602.75
[2890]	error's l1: 94.6643	error's l2: 9588.2
[2900]	error's l1: 94.587	error's l2: 9573.26
[2910]	error's l1: 94.5586	error's l2: 9567.78
[2920]	error's l1: 94.4611	error's l2: 9548.69
[2930]	error's l1: 94.4039	error's l2: 9537.59
[2940]	error's l1: 94.336	error's l2: 9524.23
[2950]	error's l1: 94.2657	error's l2: 9510.38
[2960]	error's l1: 94.1833	error's l2: 9493.76
[2970]	error's l1: 94.1162	error's l2: 9480.51
[2980]	error's l1: 93.9924	error's l2: 9456.24
[2990]	error's l1: 93.8458	error's l2: 9427.61
[3000]	error's l1: 93.7363	error's l2: 9405.92
[3010]	error's l1: 93.6545	error's l2: 9389.52
[3020]	error's l1: 93.5707	error's l2: 9372.81
[3030]	error's l1: 93.4578	error's l2: 9350.45
[3040]	error's l1: 93.3693	error's l2: 9332.68
[3050]	error's l1: 93.278	error's l2: 9314.48
[3060]	error's l1: 93.2055	error's l2: 9299.86
[3070]	error's l1: 93.1198	error's l2: 9282.61
[3080]	error's l1: 93.033	error's l2: 9265.88
[3090]	error's l1: 92.9515	error's l2: 9250.1
[3100]	error's l1: 92.9118	error's l2: 9241.93
[3110]	error's l1: 92.8696	error's l2: 9233.53
[3120]	error's l1: 92.7943	error's l2: 9218.96
[3130]	error's l1: 92.6843	error's l2: 9197.86
[3140]	error's l1: 92.5477	error's l2: 9171.69
[3150]	error's l1: 92.4999	error's l2: 9162.36
[3160]	error's l1: 92.441	error's l2: 9150.8
[3170]	error's l1: 92.3766	error's l2: 9137.96
[3180]	error's l1: 92.2501	error's l2: 9113.63
[3190]	error's l1: 92.2241	error's l2: 9109
[3200]	error's l1: 92.0995	error's l2: 9085.38
[3210]	error's l1: 92.0584	error's l2: 9077.59
[3220]	error's l1: 91.9993	error's l2: 9066.23
[3230]	error's l1: 91.8646	error's l2: 9040.7
[3240]	error's l1: 91.7854	error's l2: 9025.57
[3250]	error's l1: 91.7642	error's l2: 9021.52
[3260]	error's l1: 91.7464	error's l2: 9017.91
[3270]	error's l1: 91.7593	error's l2: 9020.44
[3280]	error's l1: 91.695	error's l2: 9008.15
[3290]	error's l1: 91.6361	error's l2: 8996.71
[3300]	error's l1: 91.546	error's l2: 8979.57
[3310]	error's l1: 91.4954	error's l2: 8969.75
[3320]	error's l1: 91.4459	error's l2: 8960.3
[3330]	error's l1: 91.359	error's l2: 8943.74
[3340]	error's l1: 91.2968	error's l2: 8931.68
[3350]	error's l1: 91.2868	error's l2: 8929.6
[3360]	error's l1: 91.2616	error's l2: 8924.74
[3370]	error's l1: 91.1918	error's l2: 8910.98
[3380]	error's l1: 91.0796	error's l2: 8889.4
[3390]	error's l1: 91.0214	error's l2: 8877.43
[3400]	error's l1: 90.9354	error's l2: 8860.89
[3410]	error's l1: 90.8362	error's l2: 8841.43
[3420]	error's l1: 90.7793	error's l2: 8829.65
[3430]	error's l1: 90.7477	error's l2: 8822.66
[3440]	error's l1: 90.6459	error's l2: 8802.9
[3450]	error's l1: 90.6012	error's l2: 8793.46
[3460]	error's l1: 90.5589	error's l2: 8784.85
[3470]	error's l1: 90.4839	error's l2: 8770.42
[3480]	error's l1: 90.4133	error's l2: 8756.39
[3490]	error's l1: 90.2844	error's l2: 8732.25
[3500]	error's l1: 90.1656	error's l2: 8709.79
[3510]	error's l1: 90.0518	error's l2: 8688.31
[3520]	error's l1: 89.9459	error's l2: 8668.2
[3530]	error's l1: 89.848	error's l2: 8649.92
[3540]	error's l1: 89.785	error's l2: 8637.97
[3550]	error's l1: 89.6802	error's l2: 8618.49
[3560]	error's l1: 89.5664	error's l2: 8597.22
[3570]	error's l1: 89.493	error's l2: 8583.62
[3580]	error's l1: 89.4334	error's l2: 8572.57
[3590]	error's l1: 89.3839	error's l2: 8563.78
[3600]	error's l1: 89.3195	error's l2: 8551.92
[3610]	error's l1: 89.269	error's l2: 8542.72
[3620]	error's l1: 89.2008	error's l2: 8530.17
[3630]	error's l1: 89.1341	error's l2: 8517.84
[3640]	error's l1: 89.0836	error's l2: 8508.5
[3650]	error's l1: 89.0345	error's l2: 8499.7
[3660]	error's l1: 88.95	error's l2: 8484.19
[3670]	error's l1: 88.8979	error's l2: 8474.23
[3680]	error's l1: 88.8468	error's l2: 8463.77
[3690]	error's l1: 88.7574	error's l2: 8446.84
[3700]	error's l1: 88.7236	error's l2: 8439.65
[3710]	error's l1: 88.6645	error's l2: 8427.89
[3720]	error's l1: 88.6332	error's l2: 8421.14
[3730]	error's l1: 88.5683	error's l2: 8408.67
[3740]	error's l1: 88.5198	error's l2: 8398.74
[3750]	error's l1: 88.4519	error's l2: 8385.8
[3760]	error's l1: 88.3616	error's l2: 8368.78
[3770]	error's l1: 88.2735	error's l2: 8351.88
[3780]	error's l1: 88.2195	error's l2: 8340.94
[3790]	error's l1: 88.1197	error's l2: 8322.4
[3800]	error's l1: 88.0275	error's l2: 8305.47
[3810]	error's l1: 87.9111	error's l2: 8284.4
[3820]	error's l1: 87.8518	error's l2: 8273.01
[3830]	error's l1: 87.7919	error's l2: 8262.09
[3840]	error's l1: 87.7132	error's l2: 8247.46
[3850]	error's l1: 87.6376	error's l2: 8233.54
[3860]	error's l1: 87.5339	error's l2: 8214.53
[3870]	error's l1: 87.4537	error's l2: 8200.03
[3880]	error's l1: 87.3929	error's l2: 8188.44
[3890]	error's l1: 87.3057	error's l2: 8172.48
[3900]	error's l1: 87.2334	error's l2: 8159.09
[3910]	error's l1: 87.1533	error's l2: 8144.47
[3920]	error's l1: 87.0692	error's l2: 8129.46
[3930]	error's l1: 86.9828	error's l2: 8113.85
[3940]	error's l1: 86.8813	error's l2: 8095.81
[3950]	error's l1: 86.8125	error's l2: 8083.26
[3960]	error's l1: 86.7164	error's l2: 8066.02
[3970]	error's l1: 86.6024	error's l2: 8045.56
[3980]	error's l1: 86.5004	error's l2: 8026.99
[3990]	error's l1: 86.411	error's l2: 8010.85
[4000]	error's l1: 86.3319	error's l2: 7996.22
[4010]	error's l1: 86.2264	error's l2: 7977.4
[4020]	error's l1: 86.1316	error's l2: 7960.18
[4030]	error's l1: 86.0494	error's l2: 7945.06
[4040]	error's l1: 85.9454	error's l2: 7926.37
[4050]	error's l1: 85.8508	error's l2: 7909.24
[4060]	error's l1: 85.7347	error's l2: 7888.67
[4070]	error's l1: 85.6797	error's l2: 7878.95
[4080]	error's l1: 85.6062	error's l2: 7865.94
[4090]	error's l1: 85.5486	error's l2: 7855.73
[4100]	error's l1: 85.4917	error's l2: 7845.53
[4110]	error's l1: 85.4153	error's l2: 7832.08
[4120]	error's l1: 85.3449	error's l2: 7819.76
[4130]	error's l1: 85.2852	error's l2: 7809.06
[4140]	error's l1: 85.2075	error's l2: 7795.48
[4150]	error's l1: 85.1467	error's l2: 7784.82
[4160]	error's l1: 85.1359	error's l2: 7782.78
[4170]	error's l1: 85.0739	error's l2: 7771.4
[4180]	error's l1: 84.9839	error's l2: 7755.45
[4190]	error's l1: 84.9425	error's l2: 7747.72
[4200]	error's l1: 84.9069	error's l2: 7740.75
[4210]	error's l1: 84.8393	error's l2: 7728.53
[4220]	error's l1: 84.7909	error's l2: 7719.65
[4230]	error's l1: 84.7532	error's l2: 7712.49
[4240]	error's l1: 84.6456	error's l2: 7693.73
[4250]	error's l1: 84.6191	error's l2: 7688.6
[4260]	error's l1: 84.5265	error's l2: 7672.37
[4270]	error's l1: 84.4629	error's l2: 7660.91
[4280]	error's l1: 84.452	error's l2: 7658.55
[4290]	error's l1: 84.3939	error's l2: 7648.32
[4300]	error's l1: 84.3594	error's l2: 7642.01
[4310]	error's l1: 84.3062	error's l2: 7632.25
[4320]	error's l1: 84.2577	error's l2: 7623.35
[4330]	error's l1: 84.2105	error's l2: 7614.95
[4340]	error's l1: 84.146	error's l2: 7603.54
[4350]	error's l1: 84.107	error's l2: 7596.06
[4360]	error's l1: 84.0166	error's l2: 7580.38
[4370]	error's l1: 83.9539	error's l2: 7569.1
[4380]	error's l1: 83.8576	error's l2: 7552.44
[4390]	error's l1: 83.7721	error's l2: 7537.63
[4400]	error's l1: 83.6454	error's l2: 7516
[4410]	error's l1: 83.5353	error's l2: 7496.91
[4420]	error's l1: 83.4426	error's l2: 7480.69
[4430]	error's l1: 83.336	error's l2: 7462.25
[4440]	error's l1: 83.2245	error's l2: 7443.12
[4450]	error's l1: 83.1133	error's l2: 7424.21
[4460]	error's l1: 83.0437	error's l2: 7412.1
[4470]	error's l1: 83.0064	error's l2: 7405.28
[4480]	error's l1: 83.0094	error's l2: 7405.29
[4490]	error's l1: 82.9812	error's l2: 7400.12
[4500]	error's l1: 82.9355	error's l2: 7392.07
[4510]	error's l1: 82.9184	error's l2: 7388.99
[4520]	error's l1: 82.8852	error's l2: 7383.01
[4530]	error's l1: 82.8549	error's l2: 7377.49
[4540]	error's l1: 82.8297	error's l2: 7372.77
[4550]	error's l1: 82.8245	error's l2: 7371.43
[4560]	error's l1: 82.7756	error's l2: 7363
[4570]	error's l1: 82.69	error's l2: 7348.64
[4580]	error's l1: 82.6043	error's l2: 7334.17
[4590]	error's l1: 82.5456	error's l2: 7324.42
[4600]	error's l1: 82.5026	error's l2: 7317.26
[4610]	error's l1: 82.4491	error's l2: 7308.42
[4620]	error's l1: 82.3872	error's l2: 7298.13
[4630]	error's l1: 82.3095	error's l2: 7285.15
[4640]	error's l1: 82.2434	error's l2: 7274.26
[4650]	error's l1: 82.2242	error's l2: 7271.2
[4660]	error's l1: 82.1763	error's l2: 7263.12
[4670]	error's l1: 82.0925	error's l2: 7249.04
[4680]	error's l1: 82.0095	error's l2: 7235.2
[4690]	error's l1: 81.9333	error's l2: 7222.35
[4700]	error's l1: 81.8759	error's l2: 7212.66
[4710]	error's l1: 81.8086	error's l2: 7201.37
[4720]	error's l1: 81.7384	error's l2: 7189.57
[4730]	error's l1: 81.6761	error's l2: 7179.05
[4740]	error's l1: 81.6231	error's l2: 7170.1
[4750]	error's l1: 81.5661	error's l2: 7160.63
[4760]	error's l1: 81.509	error's l2: 7150.48
[4770]	error's l1: 81.4485	error's l2: 7140.05
[4780]	error's l1: 81.3847	error's l2: 7128.8
[4790]	error's l1: 81.3161	error's l2: 7116.99
[4800]	error's l1: 81.2602	error's l2: 7107.09
[4810]	error's l1: 81.1998	error's l2: 7096.81
[4820]	error's l1: 81.1295	error's l2: 7084.79
[4830]	error's l1: 81.0723	error's l2: 7075.1
[4840]	error's l1: 81.0331	error's l2: 7068.29
[4850]	error's l1: 80.9704	error's l2: 7057.74
[4860]	error's l1: 80.945	error's l2: 7053.28
[4870]	error's l1: 80.9351	error's l2: 7051.49
[4880]	error's l1: 80.9075	error's l2: 7046.87
[4890]	error's l1: 80.8928	error's l2: 7044.33
[4900]	error's l1: 80.8954	error's l2: 7044.66
[4910]	error's l1: 80.8619	error's l2: 7039.04
[4920]	error's l1: 80.8641	error's l2: 7039.03
[4930]	error's l1: 80.8343	error's l2: 7033.91
[4940]	error's l1: 80.8354	error's l2: 7034.08
[4950]	error's l1: 80.8294	error's l2: 7032.76
[4960]	error's l1: 80.7733	error's l2: 7023.76
[4970]	error's l1: 80.7047	error's l2: 7012.76
[4980]	error's l1: 80.6291	error's l2: 7000.59
[4990]	error's l1: 80.5691	error's l2: 6991.01
[5000]	error's l1: 80.5044	error's l2: 6980.51
[5010]	error's l1: 80.4663	error's l2: 6974.64
[5020]	error's l1: 80.4017	error's l2: 6964.3
[5030]	error's l1: 80.3795	error's l2: 6960.95
[5040]	error's l1: 80.3328	error's l2: 6953.55
[5050]	error's l1: 80.3132	error's l2: 6950.51
[5060]	error's l1: 80.2847	error's l2: 6944.9
[5070]	error's l1: 80.2703	error's l2: 6941.67
[5080]	error's l1: 80.2346	error's l2: 6934.97
[5090]	error's l1: 80.2139	error's l2: 6931.04
[5100]	error's l1: 80.183	error's l2: 6925.36
[5110]	error's l1: 80.1565	error's l2: 6920.22
[5120]	error's l1: 80.1455	error's l2: 6917.4
[5130]	error's l1: 80.0967	error's l2: 6908.82
[5140]	error's l1: 80.0771	error's l2: 6904.7
[5150]	error's l1: 80.0596	error's l2: 6901.2
[5160]	error's l1: 80.0045	error's l2: 6892.07
[5170]	error's l1: 79.9367	error's l2: 6880.98
[5180]	error's l1: 79.8603	error's l2: 6868.23
[5190]	error's l1: 79.7656	error's l2: 6852.76
[5200]	error's l1: 79.7142	error's l2: 6844.35
[5210]	error's l1: 79.6373	error's l2: 6831.78
[5220]	error's l1: 79.5675	error's l2: 6820.44
[5230]	error's l1: 79.5163	error's l2: 6811.88
[5240]	error's l1: 79.4523	error's l2: 6801.46
[5250]	error's l1: 79.3825	error's l2: 6789.76
[5260]	error's l1: 79.3382	error's l2: 6782.13
[5270]	error's l1: 79.3023	error's l2: 6776.02
[5280]	error's l1: 79.2423	error's l2: 6765.41
[5290]	error's l1: 79.1912	error's l2: 6756.71
[5300]	error's l1: 79.1564	error's l2: 6750.73
[5310]	error's l1: 79.0984	error's l2: 6740.99
[5320]	error's l1: 79.0625	error's l2: 6734.51
[5330]	error's l1: 79.0066	error's l2: 6724.98
[5340]	error's l1: 78.971	error's l2: 6718.78
[5350]	error's l1: 78.9238	error's l2: 6711.18
[5360]	error's l1: 78.8414	error's l2: 6697.13
[5370]	error's l1: 78.7553	error's l2: 6682.49
[5380]	error's l1: 78.662	error's l2: 6666.7
[5390]	error's l1: 78.5915	error's l2: 6655.04
[5400]	error's l1: 78.5057	error's l2: 6640.46
[5410]	error's l1: 78.4435	error's l2: 6629.85
[5420]	error's l1: 78.3836	error's l2: 6619.88
[5430]	error's l1: 78.3192	error's l2: 6609.01
[5440]	error's l1: 78.2256	error's l2: 6593.34
[5450]	error's l1: 78.2203	error's l2: 6592.21
[5460]	error's l1: 78.265	error's l2: 6599.22
[5470]	error's l1: 78.3066	error's l2: 6605.92
[5480]	error's l1: 78.3871	error's l2: 6618.51
[5490]	error's l1: 78.4012	error's l2: 6620.52
[5500]	error's l1: 78.4486	error's l2: 6627.98
[5510]	error's l1: 78.4892	error's l2: 6634.64
[5520]	error's l1: 78.5328	error's l2: 6641.68
[5530]	error's l1: 78.5826	error's l2: 6649.71
[5540]	error's l1: 78.6513	error's l2: 6660.24
Early stopping, best iteration is:
[5445]	error's l1: 78.1962	error's l2: 6588.42

feat_imp = pd.Series(model.feature_importances_, index=X_train.columns)
feat_imp.nlargest(30).plot(kind='barh', figsize=(8,10))

<Axes: >

def show_staistics(model_obj, X_dataset , y_data_set):
    y_pred = model_obj.predict(X_dataset)
    mse = mean_squared_error(y_data_set, y_pred)
    
    print("The mean squared error (MSE) on test set: {:.4f}".format(mse))
    print("The mean absolute error (MAE) on test set: {:.4f}".format(mean_absolute_error(y_data_set, y_pred)))
    print("The RMSE on test set: {:.4f}".format(sqrt(mse)))
    print("The r2_score on test set: {:.4f}".format(r2_score(y_data_set, y_pred)))
    print("The smape on test set: {:.4f}".format(smape(y_data_set, y_pred)))
    print("The vape on test set: {:.4f}".format(vape(y_data_set, y_pred)))

show_staistics(model, X_test , y_test)

The mean squared error (MSE) on test set: 6588.4159
The mean absolute error (MAE) on test set: 78.1962
The RMSE on test set: 81.1691
The r2_score on test set: -3.9530
The smape on test set: 74.6261
The vape on test set: 1.0435

df['pred'] = model.predict(df.drop(['clicks', "pred", "date_obj", "date","_merge"], axis=1, errors="ignore"))
df['date_obj'] = pd.to_datetime(df['date']).dt.date
df_gp = df.groupby(['date_obj'], as_index=False).agg({
    "clicks": sum,
    "pred": sum
})

fig = px.line(df_gp, x="date_obj", y=["clicks","pred"], title='Validation Real CPC')
fig.show()

	latitude	longitude	maxtempC	mintempC	avgtempC	totalSnow_cm	sunHour	tempC	windspeedKmph	winddirDegree	...	precipMM	humidity	visibility	pressure	cloudcover	HeatIndexC	WindChillC	WindGustKmph	FeelsLikeC	uvIndex
count	0.0	0.0	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	...	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000	43056.000000
mean	NaN	NaN	9.016211	-0.179487	5.096386	0.353349	10.226709	4.158305	11.882130	213.241708	...	0.053881	69.241662	9.033422	1015.371191	41.293107	4.285628	1.504645	19.061014	1.530147	1.978632
std	NaN	NaN	12.763126	10.719672	11.858362	1.333326	3.939936	12.005125	6.018685	91.218255	...	0.286275	18.626919	2.275441	9.429675	34.606166	12.108482	14.055261	9.509353	14.093446	1.574934
min	NaN	NaN	-28.000000	-34.000000	-31.000000	0.000000	2.000000	-34.000000	0.000000	0.000000	...	0.000000	8.000000	0.000000	977.000000	0.000000	-34.000000	-44.000000	0.000000	-44.000000	1.000000
25%	NaN	NaN	0.000000	-7.000000	-3.000000	0.000000	6.900000	-4.000000	8.000000	148.000000	...	0.000000	56.000000	10.000000	1009.000000	10.000000	-4.000000	-8.000000	12.000000	-8.000000	1.000000
50%	NaN	NaN	9.000000	1.000000	6.000000	0.000000	10.300000	5.000000	11.000000	233.000000	...	0.000000	72.000000	10.000000	1015.000000	31.000000	5.000000	2.000000	18.000000	2.000000	1.000000
75%	NaN	NaN	19.000000	9.000000	15.000000	0.000000	13.500000	13.000000	15.000000	282.000000	...	0.000000	84.000000	10.000000	1021.000000	73.000000	13.000000	12.000000	24.000000	12.000000	2.000000
max	NaN	NaN	38.000000	21.000000	32.000000	18.100000	16.300000	38.000000	51.000000	360.000000	...	12.100000	100.000000	10.000000	1061.000000	100.000000	38.000000	38.000000	94.000000	38.000000	9.000000

	city	region	country	latitude	longitude	datetime	maxtempC	mintempC	avgtempC	totalSnow_cm	...	humidity	visibility	pressure	cloudcover	HeatIndexC	WindChillC	WindGustKmph	FeelsLikeC	uvIndex	date
0	Calgary	Alberta	Canada	NaN	NaN	2018-04-01 00:00:00	-8	-14	-10	0.1	...	87	10	1029	14	-13	-17	13	-17	1	2018-04-01
1	Calgary	Alberta	Canada	NaN	NaN	2018-04-01 01:00:00	-8	-14	-10	0.1	...	86	10	1028	25	-13	-16	11	-16	1	2018-04-01
2	Calgary	Alberta	Canada	NaN	NaN	2018-04-01 02:00:00	-8	-14	-10	0.1	...	85	10	1028	36	-13	-14	8	-14	1	2018-04-01
3	Calgary	Alberta	Canada	NaN	NaN	2018-04-01 03:00:00	-8	-14	-10	0.1	...	84	10	1027	47	-13	-13	6	-13	1	2018-04-01
4	Calgary	Alberta	Canada	NaN	NaN	2018-04-01 04:00:00	-8	-14	-10	0.1	...	84	10	1026	62	-12	-12	6	-12	1	2018-04-01

	maxtempC	mintempC	avgtempC	totalSnow_cm	sunHour	tempC	windspeedKmph	winddirDegree	weatherCode	precipMM	humidity	visibility	pressure	cloudcover	HeatIndexC	WindChillC	WindGustKmph	FeelsLikeC	uvIndex	date
0	-8	-14	-10	0.1	6.5	-14	6	219	113	0.0	87	10	1029	14	-13	-17	13	-17	1	2018-04-01
1	-8	-14	-10	0.1	6.5	-14	5	221	113	0.0	86	10	1028	25	-13	-16	11	-16	1	2018-04-01
2	-8	-14	-10	0.1	6.5	-14	4	222	116	0.0	85	10	1028	36	-13	-14	8	-14	1	2018-04-01
3	-8	-14	-10	0.1	6.5	-13	3	223	116	0.0	84	10	1027	47	-13	-13	6	-13	1	2018-04-01
4	-8	-14	-10	0.1	6.5	-13	3	173	116	0.0	84	10	1026	62	-12	-12	6	-12	1	2018-04-01
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
43051	-4	-19	-11	0.0	10.0	-11	7	130	260	0.0	97	0	1007	66	-11	-16	14	-16	1	2023-02-27
43052	-4	-19	-11	0.0	10.0	-16	4	156	260	0.0	100	0	1009	95	-16	-19	8	-19	1	2023-02-27
43053	-4	-19	-11	0.0	10.0	-17	3	200	260	0.0	100	0	1011	100	-17	-20	7	-20	1	2023-02-27
43054	-4	-19	-11	0.0	10.0	-18	4	210	260	0.0	100	0	1012	100	-18	-21	8	-21	1	2023-02-27
43055	-4	-19	-11	0.0	10.0	-18	4	240	260	0.0	100	0	1012	100	-18	-22	9	-22	1	2023-02-27

	month_dt	day_in_month_dt	day_of_week_dt	week_in_year_dt	is_holiday	day_in_year_dt	clicks	bidder_impact	new_befaviour	maxtempC	...	precipMM	humidity	visibility	pressure	cloudcover	HeatIndexC	WindChillC	WindGustKmph	FeelsLikeC	uvIndex
month_dt	1.000000	0.008001	-0.000809	0.967513	0.019695	0.996412	-0.190993	0.044454	0.150674	0.148657	...	-0.010382	-0.090976	0.043537	-0.069565	-0.144789	0.145129	0.149505	-0.023335	0.149214	0.132562
day_in_month_dt	0.008001	1.000000	0.001159	0.063113	-0.065674	0.092327	0.002772	0.000098	-0.003636	0.023674	...	-0.009181	-0.010156	0.037998	-0.001840	-0.036714	0.026638	0.025228	0.031129	0.025632	0.015899
day_of_week_dt	-0.000809	0.001159	1.000000	0.002069	-0.148241	-0.000705	-0.318300	-0.001786	0.000098	0.006618	...	0.045648	0.005289	-0.011810	-0.025675	0.019395	0.008815	0.005601	0.049326	0.005504	-0.004704
week_in_year_dt	0.967513	0.063113	0.002069	1.000000	0.048629	0.968883	-0.191577	0.048283	0.147267	0.141697	...	-0.012141	-0.084251	0.048870	-0.078749	-0.150955	0.137343	0.140783	-0.005155	0.140535	0.126590
is_holiday	0.019695	-0.065674	-0.148241	0.048629	1.000000	0.014155	-0.026664	0.019173	0.005456	-0.057459	...	0.048889	0.041050	-0.039205	0.004391	0.047410	-0.057342	-0.056484	-0.024152	-0.056412	-0.035651
day_in_year_dt	0.996412	0.092327	-0.000705	0.968883	0.014155	1.000000	-0.190320	0.044234	0.149554	0.147168	...	-0.011444	-0.089604	0.045515	-0.068213	-0.146330	0.143854	0.148062	-0.019459	0.147806	0.131018
clicks	-0.190993	0.002772	-0.318300	-0.191577	-0.026664	-0.190320	1.000000	-0.210558	-0.443571	0.021499	...	0.059416	-0.052855	0.013200	-0.015246	0.056980	0.035576	0.035221	-0.086451	0.035937	0.042409
bidder_impact	0.044454	0.000098	-0.001786	0.048283	0.019173	0.044234	-0.210558	1.000000	0.417844	-0.060546	...	-0.136919	-0.017700	-0.048229	-0.021034	0.048378	-0.094595	-0.104631	0.108024	-0.104525	-0.057091
new_befaviour	0.150674	-0.003636	0.000098	0.147267	0.005456	0.149554	-0.443571	0.417844	1.000000	0.070994	...	-0.082573	0.053911	-0.049675	0.030904	0.020307	0.034494	0.031208	-0.016830	0.031170	0.092718
maxtempC	0.148657	0.023674	0.006618	0.141697	-0.057459	0.147168	0.021499	-0.060546	0.070994	1.000000	...	0.060175	-0.673511	0.526831	-0.492112	-0.526465	0.987673	0.986609	-0.084348	0.986489	0.904912
mintempC	0.159483	0.029963	0.012383	0.150150	-0.058273	0.158466	0.034269	-0.112628	0.009503	0.956398	...	0.135400	-0.599872	0.481568	-0.539485	-0.410667	0.985379	0.983360	-0.048927	0.983301	0.823329
avgtempC	0.147422	0.024553	0.007521	0.139544	-0.056675	0.145972	0.028318	-0.086849	0.046233	0.992044	...	0.092302	-0.647258	0.506180	-0.515709	-0.478202	0.998185	0.996153	-0.069044	0.996077	0.884962
totalSnow_cm	-0.024360	-0.020358	0.040147	-0.028003	0.002067	-0.025670	0.016985	0.021353	-0.071795	-0.239932	...	0.275356	0.297425	-0.538219	0.063721	0.371428	-0.210476	-0.214728	-0.037912	-0.214724	-0.288111
sunHour	-0.091664	0.006731	-0.011023	-0.098925	-0.057076	-0.094303	0.112869	-0.119186	-0.022630	0.776056	...	0.095189	-0.496166	0.455507	-0.284653	-0.483217	0.764808	0.766930	-0.148582	0.767750	0.780169
tempC	0.147565	0.026401	0.009094	0.139641	-0.057624	0.146253	0.031268	-0.094212	0.036373	0.987946	...	0.104282	-0.639014	0.500427	-0.522312	-0.464384	0.999744	0.997478	-0.064509	0.997429	0.875601
windspeedKmph	-0.064031	0.015223	0.048207	-0.053884	-0.027168	-0.062482	0.019540	0.078734	-0.046249	0.001629	...	0.172958	-0.073503	-0.018422	-0.185487	0.106291	0.040012	-0.010843	0.889979	-0.011115	-0.085789
winddirDegree	0.055624	0.034317	-0.007678	0.066043	-0.031765	0.059476	-0.131122	0.064972	0.043331	-0.122194	...	-0.057659	-0.063403	0.116067	-0.057722	-0.152231	-0.134560	-0.150942	0.430677	-0.151810	-0.120092
weatherCode	-0.143989	-0.032262	0.022461	-0.150448	0.074485	-0.145634	0.061972	-0.093022	-0.106947	-0.350675	...	0.469313	0.574534	-0.700698	0.078764	0.760827	-0.290144	-0.292403	-0.057592	-0.292213	-0.401686
precipMM	-0.010382	-0.009181	0.045648	-0.012141	0.048889	-0.011444	0.059416	-0.136919	-0.082573	0.060175	...	1.000000	0.224709	-0.245414	-0.132822	0.295468	0.105652	0.100826	0.078465	0.100930	-0.022049
humidity	-0.090976	-0.010156	0.005289	-0.084251	0.041050	-0.089604	-0.052855	-0.017700	0.053911	-0.673511	...	0.224709	1.000000	-0.608430	0.304383	0.674826	-0.635855	-0.629335	-0.065004	-0.627590	-0.647113
visibility	0.043537	0.037998	-0.011810	0.048870	-0.039205	0.045515	0.013200	-0.048229	-0.049675	0.526831	...	-0.245414	-0.608430	1.000000	-0.263557	-0.678692	0.499646	0.500905	0.073652	0.500531	0.490655
pressure	-0.069565	-0.001840	-0.025675	-0.078749	0.004391	-0.068213	-0.015246	-0.021034	0.030904	-0.492112	...	-0.132822	0.304383	-0.263557	1.000000	0.157952	-0.521640	-0.517412	-0.171832	-0.516869	-0.345647
cloudcover	-0.144789	-0.036714	0.019395	-0.150955	0.047410	-0.146330	0.056980	0.048378	0.020307	-0.526465	...	0.295468	0.674826	-0.678692	0.157952	1.000000	-0.463685	-0.466139	-0.043838	-0.466146	-0.593944
HeatIndexC	0.145129	0.026638	0.008815	0.137343	-0.057342	0.143854	0.035576	-0.094595	0.034494	0.987673	...	0.105652	-0.635855	0.499646	-0.521640	-0.463685	1.000000	0.997492	-0.066581	0.997584	0.876820
WindChillC	0.149505	0.025228	0.005601	0.140783	-0.056484	0.148062	0.035221	-0.104631	0.031208	0.986609	...	0.100826	-0.629335	0.500905	-0.517412	-0.466139	0.997492	1.000000	-0.115995	0.999950	0.876609
WindGustKmph	-0.023335	0.031129	0.049326	-0.005155	-0.024152	-0.019459	-0.086451	0.108024	-0.016830	-0.084348	...	0.078465	-0.065004	0.073652	-0.171832	-0.043838	-0.066581	-0.115995	1.000000	-0.116300	-0.157505
FeelsLikeC	0.149214	0.025632	0.005504	0.140535	-0.056412	0.147806	0.035937	-0.104525	0.031170	0.986489	...	0.100930	-0.627590	0.500531	-0.516869	-0.466146	0.997584	0.999950	-0.116300	1.000000	0.877241
uvIndex	0.132562	0.015899	-0.004704	0.126590	-0.035651	0.131018	0.042409	-0.057091	0.092718	0.904912	...	-0.022049	-0.647113	0.490655	-0.345647	-0.593944	0.876820	0.876609	-0.157505	0.877241	1.000000

	date	month_dt	day_in_month_dt	day_of_week_dt	week_in_year_dt	is_holiday	day_in_year_dt	clicks	bidder_impact	new_befaviour
0	2018-01-01	1	1	0	1	True	1	112	False	False
1	2018-01-02	1	2	1	1	False	2	216	False	False
2	2018-01-03	1	3	2	1	False	3	167	False	False
3	2018-01-04	1	4	3	1	False	4	175	False	False
4	2018-01-05	1	5	4	1	False	5	158	False	False